package com.haozhen.rdd

/**
  * @author haozhen
  * @email haozh@ync1.com
  * @date 2021/1/3  0:15
  */
object WordCount {

  def main(args: Array[String]): Unit = {
    import org.apache.spark.{SparkConf, SparkContext}
    //1创建SparkContext

    val conf = new SparkConf().setMaster("local[*]").setAppName("wordCount")
    val sc = new SparkContext(conf)
    //2读取本地文件
    sc.setLogLevel("WARN")
    val lines = sc.textFile("data/words.txt")
    //rdd转换

    val words = lines.flatMap(line=>line.split("\\s+"))

    val wordMap = words.map(x=>(x,1))

    val wordReduce = wordMap.reduceByKey(_+_)
    //4s输出
    wordReduce.saveAsTextFile("output")
    //关闭SparkContext
    sc.stop()
    //打包 使用spark-submit提交集群
  }

}
